# -*- coding: utf-8 -*-
import scrapy,random,json
from bili.phone_agent import agent
from bili.items import BiliItem
from bili.redisfilter import BloomFilter

class BiliSpider(scrapy.Spider):
    name = 'Bili'
    custom_settings = {
        'DEFAULT_REQUEST_HEADERS':{
            "Host": "api.bilibili.com",
            "Origin": "https://space.bilibili.com",
            "User-Agent": random.choice(agent),
            "Cookie": "finger=41b1091e; buvid3=D5700933-EBD6-49E7-9ECC-41B9A9FB14D56702infoc",
            "X-Requested-With": "tv.danmaku.bili",
        }
    }
    fans_url='https://api.bilibili.com/x/relation/followers?vmid=%s'
    follows='https://api.bilibili.com/x/relation/followings?vmid=%s'
    def start_requests(self):
        # 种子ID
        uid=39202427
        # 伪装 referer
        headers={
            'Referer':'https://space.bilibili.com/h5/follow?mid=%s&type=fans&url_from_h5=1' % uid
        }
        # 种子选手
        start_fans_url=self.fans_url % uid
        start_follows_url=self.follows % uid
        yield scrapy.Request(start_fans_url,headers=headers,callback=self.fans_list)
        yield scrapy.Request(start_follows_url,headers=headers,callback=self.fans_list)
    # 通过种子用户获取粉丝和关注列表
    def fans_list(self,response):
        html_fans_list=response.text
        fans_list_dict=json.loads(html_fans_list)
        list_fans=fans_list_dict.get('data').get('list')
        if list_fans:
            for fans in list_fans:
                # 粉丝id
                mid=fans.get('mid')
                # 伪装 referer
                headers = {
                    'Referer': 'https://space.bilibili.com/h5/%s?height=50' % mid
                }
                # 请求详情
                fans_info='https://api.bilibili.com/x/space/app/index?mid=%s&platform=android' % mid
                bf=BloomFilter()
                if bf.isContains(fans_info):  # 判断字符串是否存在
                    print('exists!')
                else:
                    print('not exists!')
                    bf.insert(fans_info)
                    yield scrapy.Request(fans_info,callback=self.user_info)
    def user_info(self,resposne):
        item = BiliItem()

        html_user = resposne.text
        html_user_dict = json.loads(html_user)['data']['info']
        if html_user_dict:
            umid = html_user_dict['mid']
            item["user_id"] = html_user_dict['mid']
            item["username"] = html_user_dict.get('name')
            item["sex"] = html_user_dict.get('sex')
            item["sign"] = html_user_dict.get('sign','') # 签名
            item["levels"] = html_user_dict.get('level','0') # 等级
            item["following"] = html_user_dict.get('following','0') # 关注
            item["follower"] = html_user_dict.get('follower','0') # 粉丝

            yield item

            fans_urls = self.fans_url % umid
            follows_urls = self.follows % umid
            headers={
                'Referer': 'https://space.bilibili.com/h5/follow?mid=%s&type=fans&url_from_h5=1' % umid
            }
            yield scrapy.Request(fans_urls,headers=headers,callback=self.fans_list)
            yield scrapy.Request(follows_urls,headers=headers,callback=self.fans_list)

